# -*- coding: utf-8 -*-
"""
Created on Wed Nov 23 13:33:48 2022

@author: sthgy
"""


import os
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
from scipy.stats import chi2
from scipy.spatial.distance import mahalanobis

from utils import Fund
from utils import get_fund_list, date_format

plt.rcParams["font.sans-serif"]=["SimHei"] #设置字体
plt.rcParams["axes.unicode_minus"]=False #该语句解决图像中的“-”负号的乱码问题
plt.rcParams.update({"font.size":20})#此处必须添加此句代码方可改变标题字体大小

def sift():
    ''' 筛选基金
    0. 只包含股票型（含指数）、债券型、绝对收益，不包含商品/FOF/REITs/QDII
    1. 存续期覆盖2017-01-01至2021-12-31
    2. （最近5个季度平均）规模不小于5亿
    3. 避免理财、定期开放型基金带来申购上的问题
    4. 收益-波动与类型均值的距离不属于outlier，避免建仓期/大额申赎/拆分/合并/异常值的影响
    '''
    fund_list = get_fund_list() # 已经过滤不合适的类型
    funds = {}
    for _, row in fund_list.iterrows():
        print(row['基金代码'])
        fund = Fund(
            fund_code=row['基金代码'],
            start_date=date_format('2017-01-01'),
            end_date=date_format('2021-12-31'),
            fund_name=row['基金简称'],
            fund_type=row['基金类型']
            )
        funds[row['基金代码']] = fund
    df = pd.DataFrame()
    df['基金代码'] = [x.fund_code for x in funds.values()]
    df['基金名称'] = [x.fund_name for x in funds.values()]
    df['基金类型'] = [x.fund_type for x in funds.values()]
    df['基金规模'] = [x.fund_scale for x in funds.values()]
    df['净值起始日期'] = [x.start_date for x in funds.values()]
    df['净值终止日期'] = [x.end_date for x in funds.values()]
    df['年化收益率'] = [x.annual_return for x in funds.values()]
    df['年化波动率'] = [x.annual_volatility for x in funds.values()]
    # 过滤不合适的存续期
    sifted = df.loc[
        (df['净值起始日期'] <= date_format('2017-01-01')) &
        (df['净值终止日期'] >= date_format('2021-12-31')) &
        (df['年化收益率'].notnull()),
        :].reset_index(drop=True)
    # 过滤理财和定开
    sifted = sifted.loc[
        (sifted['基金名称'].apply(
            lambda x: '理财' not in x and '定开' not in x and '定期' not in x)
            ), :].reset_index(drop=True)
    # 过滤规模太小的
    sifted = sifted.loc[(sifted['基金规模'] > 5), :].reset_index(drop=True)
    # 确定收益-波动的outlier（并不是业内通行的方式，是我自己觉得好实现瞎编的）
    # # 欧氏距离
    # grouped = sifted.groupby(
    #     by=['基金类型'], as_index=False
    #     ).agg({'年化收益率':np.mean,'年化波动率':np.std})
    # sifted = pd.merge(
    #     sifted, grouped, on=['基金类型'],
    #     how='left', suffixes=['', '组平均'])
    # sifted['距离'] = (
    #     (sifted['年化收益率'] - sifted['年化收益率组平均'])**2 +
    #     (sifted['年化波动率'] - sifted['年化波动率组平均'])**2
    #     ).apply(np.sqrt)
    # grouped_m = sifted.groupby(
    #     by=['基金类型'], as_index=False
    #     ).agg({'距离':np.mean}).rename(columns={'距离':'距离平均'})
    # grouped_s = sifted.groupby(
    #     by=['基金类型'], as_index=False
    #     ).agg({'距离':np.std}).rename(columns={'距离':'距离标准差'})
    # grouped_d = pd.merge(grouped_m, grouped_s, on=['基金类型'])
    # sifted = pd.merge(sifted, grouped_d, on=['基金类型'], how='left')
    # sifted['距离上限'] = sifted['距离平均'] + 3 * sifted['距离标准差']
    # sifted = sifted.loc[
    #     (sifted['距离'] < sifted['距离上限']),
    #     :].reset_index(drop=True)
    
    # 收益率-波动率的马氏距离
    data_m_list = []
    for group, data in sifted.groupby(by=['基金类型']):
        iv = np.linalg.inv(data[['年化收益率','年化波动率']].cov())
        mean_mean = data['年化收益率'].mean()
        std_mean = data['年化波动率'].mean()
        data_m = data.copy()
        data_m['马氏距离'] = data_m[['年化收益率','年化波动率']].apply(
            lambda x: mahalanobis(x, [mean_mean, std_mean], iv), axis=1)
        data_m['卡方分位数'] = data_m['马氏距离'].apply(
            lambda x: 1-chi2.cdf(x**2, 1))
        data_m_list.append(data_m)
    sifted_m = pd.concat(data_m_list, ignore_index=True)
    sifted_m = sifted_m.loc[
        (sifted_m['卡方分位数']>0.01),
        :].reset_index(drop=True)
    funds_sifted = {
        x:y for x, y in funds.items()
        if x in sifted_m['基金代码'].values}
    dict2df = pd.Series(funds_sifted)
    with pd.HDFStore('./FundData/data.h5') as hdf_file:
        dict2df.to_hdf(hdf_file, 'funds_sift')

def read_funds_sifted():
    ''' 读取筛选过的基金们 '''
    with pd.HDFStore('./FundData/data.h5') as hdf_file:
        fund_series = pd.read_hdf(hdf_file, 'funds_sifted')
    return {x:y for x,y in fund_series.items()}

# def riskfree_modify(riskfree=0.02):
#     ''' 无风险利率修正
#     参考近期shibor的平均利率，实际无风险利率会变动，这里视为常数
#     但是后续通过限制最低收益来体现，没有采用此种方法'''
#     with pd.HDFStore('./FundData/data.h5') as hdf_file:
#         fund_series = pd.read_hdf(hdf_file, 'funds_sifted')
#     modified = {x:Fund(
#         fund_code=x,
#         start_date=date_format('2017-01-01'),
#         end_date=date_format('2021-12-31'),
#         fund_name=y.fund_name,
#         fund_type=y.fund_type,
#         riskfree=riskfree
#         ) for x, y in fund_series.items()}
#     dict2df = pd.Series(modified)
#     with pd.HDFStore('./FundData/data.h5') as hdf_file:
#         dict2df.to_hdf(hdf_file, 'funds_modified')

def describe():
    ''' 汇总 '''
    funds = read_funds_sifted().values()
    description = pd.DataFrame({
        '基金代码':[x.fund_code for x in funds],
        '基金名称':[x.fund_name for x in funds],
        '基金类型':[x.fund_type for x in funds],
        '基金规模':[x.fund_scale for x in funds],
        '净值起始日期':[x.start_date for x in funds],
        '净值终止日期':[x.end_date for x in funds],
        '年化收益率':[x.annual_return for x in funds],
        '年化波动率':[x.annual_volatility for x in funds],
        '年化下行波动率':[x.annual_dd_vol for x in funds],
        '最大回撤':[x.drawdown for x in funds],
        '最大回撤期起始日':[x.drawdown_start for x in funds],
        '最大回撤期终止日':[x.drawdown_end for x in funds],
        '夏普比率':[x.sharpe for x in funds],
        '所提诺比率':[x.sortino for x in funds],
        '卡玛比率':[x.calmar for x in funds]
        })[[
            '基金代码',
            '基金名称',
            '基金类型',
            '基金规模',
            '净值起始日期',
            '净值终止日期',
            '年化收益率',
            '年化波动率',
            '年化下行波动率',
            '最大回撤',
            '最大回撤期起始日',
            '最大回撤期终止日',
            '夏普比率',
            '所提诺比率',
            '卡玛比率'
            ]]
    with pd.HDFStore('./FundData/data.h5') as hdf_file:
        description.to_hdf(hdf_file, 'description')

def plot():
    ''' 画图 '''
    with pd.HDFStore('./FundData/data.h5','r') as hdf_file:
        sifted = pd.read_hdf(hdf_file, 'description')
    # 筛选基金规模（平均）在5亿以上
    sifted5 = sifted.loc[sifted['基金规模']>5,:].reset_index(drop=True)
    x = sifted5['年化收益率']
    y = sifted5['年化波动率']
    # 散点图+分布图
    grouped = sifted5.groupby(
        by=['基金类型'], as_index=False
        ).agg({'年化波动率':np.mean})
    grouped['类型排序'] = grouped['年化波动率'].rank()
    for_scatter = pd.merge(
        sifted5[['基金类型','基金规模','年化收益率','年化波动率']],
        grouped[['基金类型','类型排序']],
        how='left', on=['基金类型'])
    fig = plt.figure(figsize=(20, 20))
    gs = fig.add_gridspec(2, 2,  width_ratios=(4, 1), height_ratios=(1, 4),
                          left=0.1, right=0.9, bottom=0.1, top=0.9,
                          wspace=0.05, hspace=0.05)
    ax = fig.add_subplot(gs[1, 0])
    ax_histx = fig.add_subplot(gs[0, 0], sharex=ax)
    ax_histy = fig.add_subplot(gs[1, 1], sharey=ax)
    ax_histx.tick_params(axis="x", labelbottom=False)
    ax_histy.tick_params(axis="y", labelleft=False)
    colors = plt.get_cmap('rainbow')(np.linspace(0,1,grouped.shape[0]))
    for fund_type, data in for_scatter.groupby(['基金类型']):
        ax.scatter(
            x=data['年化收益率'],
            y=data['年化波动率'],
            c=data['类型排序'].apply(lambda x: colors[int(x)-1]),
            s=20,
            label=fund_type,
            alpha=0.6,
            edgecolors='none')
    width_x = (x.max() - x.min())/200
    width_y = (y.max() - y.min())/200
    bins_x = np.arange(x.min()-3*width_x, x.max()+3*width_x, width_x)
    bins_y = np.arange(y.min()-3*width_y, y.max()+3*width_y, width_y)
    ax_histx.hist(x, bins=bins_x)
    ax_histy.hist(y, bins=bins_y, orientation='horizontal')
    ax.legend()
    ax.set_xlabel('年化收益率')
    ax.set_ylabel('年化波动率')
    ax.grid(True)
    # plt.show()
    if not os.path.exists('./Image'):
        os.makedirs('./Image')
    plt.savefig('./Image/Scatter+Histogram.jpg')
    plt.close()

if __name__ == '__main__':
    # describe()
    # plot()
    pass